/****************************************************************************
 * kernel/irq/irq_csection.c
 *
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.  The
 * ASF licenses this file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the
 * License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 ****************************************************************************/

/****************************************************************************
 * Included Files
 ****************************************************************************/

#include <tinyara/config.h>

#include <sys/types.h>
#include <assert.h>

#include <tinyara/init.h>
#include <tinyara/spinlock.h>
#include <tinyara/sched_note.h>
#include <arch/irq.h>

#include "sched/sched.h"
#include "irq/irq.h"

#ifdef CONFIG_IRQCOUNT

/****************************************************************************
 * Public Data
 ****************************************************************************/

#ifdef CONFIG_SMP
/* This is the spinlock that enforces critical sections when interrupts are
 * disabled.
 */

volatile spinlock_t g_cpu_irqlock = SP_UNLOCKED;

/* Used to keep track of which CPU(s) hold the IRQ lock. */

volatile spinlock_t g_cpu_irqsetlock;
volatile cpu_set_t g_cpu_irqset;

/* Handles nested calls to enter_critical section from interrupt handlers */

volatile uint8_t g_cpu_nestcount[CONFIG_SMP_NCPUS];
#endif

/****************************************************************************
 * Private Functions
 ****************************************************************************/

/****************************************************************************
 * Name: irq_waitlock
 *
 * Description:
 *   Spin to get g_cpu_irqlock, handling a known deadlock condition:
 *
 *   A deadlock may occur if enter_critical_section is called from an
 *   interrupt handler.  Suppose:
 *
 *   - CPUn is in a critical section and has the g_cpu_irqlock spinlock.
 *   - CPUm takes an interrupt and attempts to enter the critical section.
 *   - It spins waiting on g_cpu_irqlock with interrupts disabled.
 *   - CPUn calls up_cpu_pause() to pause operation on CPUm.  This will
 *     issue an inter-CPU interrupt to CPUm
 *   - But interrupts are disabled on CPUm so the up_cpu_pause() is never
 *     handled, causing the deadlock.
 *
 *   This same deadlock can occur in the normal tasking case:
 *
 *   - A task on CPUn enters a critical section and has the g_cpu_irqlock
 *     spinlock.
 *   - Another task on CPUm attempts to enter the critical section but has
 *     to wait, spinning to get g_cpu_irqlock with interrupts disabled.
 *   - The task on CPUn causes a new task to become ready-to-run and the
 *     scheduler selects CPUm.  CPUm is requested to pause via a pause
 *     interrupt.
 *   - But the task on CPUm is also attempting to enter the critical
 *     section.  Since it is spinning with interrupts disabled, CPUm cannot
 *     process the pending pause interrupt, causing the deadlock.
 *
 *   This function detects this deadlock condition while spinning with
 *   interrupts disabled.
 *
 * Input Parameters:
 *   cpu - The index of CPU that is trying to enter the critical section.
 *
 * Returned Value:
 *   True:  The g_cpu_irqlock spinlock has been taken.
 *   False: The g_cpu_irqlock spinlock has not been taken yet, but there is
 *          a pending pause interrupt request.
 *
 ****************************************************************************/

#ifdef CONFIG_SMP
static bool irq_waitlock(int cpu)
{
#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS
	FAR struct tcb_s *tcb = current_task(cpu);

	/* Notify that we are waiting for a spinlock */

	sched_note_spinlock(tcb, &g_cpu_irqlock);
#endif

	/* Duplicate the spin_lock() logic from spinlock.c, but adding the check
	 * for the deadlock condition.
	 */

	while (spin_trylock_wo_note(&g_cpu_irqlock) == SP_LOCKED) {
		/* Is a pause request pending? */

		if (up_cpu_pausereq(cpu)) {
			/* Yes.. some other CPU is requesting to pause this CPU!
			 * Abort the wait and return false.
			 */

#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS
			/* Notify that we have aborted the wait for the spinlock */

			sched_note_spinabort(tcb, &g_cpu_irqlock);
#endif

			return false;
		}
	}

	/* We have g_cpu_irqlock! */

#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS
	/* Notify that we have the spinlock */

	sched_note_spinlocked(tcb, &g_cpu_irqlock);
#endif

	return true;
}
#endif

/****************************************************************************
 * Public Functions
 ****************************************************************************/

/****************************************************************************
 * Name: enter_critical_section
 *
 * Description:
 *   Take the CPU IRQ lock and disable interrupts on all CPUs.  A thread-
 *   specific counter is incremented to indicate that the thread has IRQs
 *   disabled and to support nested calls to enter_critical_section().
 *
 ****************************************************************************/

#ifdef CONFIG_SMP
irqstate_t enter_critical_section(void)
{
	FAR struct tcb_s *rtcb;
	irqstate_t ret;
	int cpu;

	/* Disable interrupts.
	 *
	 * NOTE 1: Ideally this should disable interrupts on all CPUs, but most
	 * architectures only support disabling interrupts on the local CPU.
	 * NOTE 2: Interrupts may already be disabled, but we call enter_critical_section()
	 * unconditionally because we need to return valid interrupt status in any
	 * event.
	 * NOTE 3: We disable local interrupts BEFORE taking the spinlock in order
	 * to prevent possible waits on the spinlock from interrupt handling on
	 * the local CPU.
	 */

try_again:

	ret = irqsave();

	/* Verify that the system has sufficiently initialized so that the task
	 * lists are valid.
	 */

	if (g_os_initstate >= OSINIT_TASKLISTS) {
		/* If called from an interrupt handler, then just take the spinlock.
		 * If we are already in a critical section, this will lock the CPU
		 * in the interrupt handler.  Sounds worse than it is.
		 */

	if (up_interrupt_context()) {
		/* We are in an interrupt handler.  How can this happen?
		 *
		 *   1. We were not in a critical section when the interrupt
		 *      occurred.  In this case, the interrupt was entered with:
		 *
		 *      g_cpu_irqlock = SP_UNLOCKED.
		 *      g_cpu_nestcount = 0
		 *      All CPU bits in g_cpu_irqset should be zero
		 *
		 *   2. We were in a critical section and interrupts on this
		 *      this CPU were disabled -- this is an impossible case.
		 *
		 *   3. We were in critical section, but enter_critical_section() only
		 *      disabled local interrupts on a different CPU;
		 *      Interrupts could still be enabled on this CPU.
		 *
		 *      g_cpu_irqlock = SP_LOCKED.
		 *      g_cpu_nestcount = 0
		 *      The bit in g_cpu_irqset for this CPU should be zero
		 *
		 *   4. An extension of 3 is that we may be re-entered numerous
		 *      times from the same interrupt handler.  In that case:
		 *
		 *      g_cpu_irqlock = SP_LOCKED.
		 *      g_cpu_nestcount > 0
		 *      The bit in g_cpu_irqset for this CPU should be zero
		 *
		 * NOTE: However, the interrupt entry conditions can change due
		 * to previous processing by the interrupt handler that may
		 * instantiate a new thread that has irqcount > 0 and may then
		 * set the bit in g_cpu_irqset and g_cpu_irqlock = SP_LOCKED
		 */

		/* Handle nested calls to enter_critical_section() from the same
		 * interrupt.
		 */

		cpu = this_cpu();
		if (g_cpu_nestcount[cpu] > 0) {
			DEBUGASSERT(spin_islocked(&g_cpu_irqlock) && \
					g_cpu_nestcount[cpu] < UINT8_MAX);
			g_cpu_nestcount[cpu]++;

			/* This is the first call to enter_critical_section from the
			 * interrupt handler.
			 */
		} else {
			int paused = false;
			/* Make sure that the g_cpu_irqset was not already set
			 * by previous logic on this CPU that was executed by the
			 * interrupt handler.  We know that the bit in g_cpu_irqset
			 * for this CPU was zero on entry into the interrupt handler,
			 * so if it is non-zero now then we know that was the case.
			 */

			if ((g_cpu_irqset & (1 << cpu)) == 0) {
				/* Wait until we can get the spinlock (meaning that we are
				 * no longer blocked by the critical section).
				 */

try_again_in_irq:
				if (!irq_waitlock(cpu)) {
					/* We are in a deadlock condition due to a pending
					 * pause request interrupt.  Break the deadlock by
					 * handling the pause request now.
					 */
				if (!paused)
				{
					up_cpu_paused_save();
				}

					DEBUGVERIFY(up_cpu_paused(cpu));
				paused = true;

					/* NOTE: As the result of up_cpu_paused(cpu), this CPU
					 * might set g_cpu_irqset in sched_resume_scheduler()
					 * However, another CPU might hold g_cpu_irqlock.
					 * To avoid this situation, releae g_cpu_irqlock first.
					 */

					if ((g_cpu_irqset & (1 << cpu)) != 0) {
						spin_clrbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, \
								&g_cpu_irqlock);
					}

					/* NOTE: Here, this CPU does not hold g_cpu_irqlock,
					 * so call irq_waitlock(cpu) to acquire g_cpu_irqlock.
					 */

					goto try_again_in_irq;
				}
			}

			/* In any event, the nesting count is now one */

			g_cpu_nestcount[cpu] = 1;

			/* Also set the CPU bit so that other CPUs will be aware that
			 * this CPU holds the critical section.
			 */

			spin_setbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, \
					&g_cpu_irqlock);
			if (paused)
			{
				up_cpu_paused_restore();
			}
		}
	} else {
		/* Normal tasking environment.
		 *
		 * Get the TCB of the currently executing task on this CPU (avoid
		 * using this_task() which can recurse.
		 */

		cpu = this_cpu();
		rtcb = current_task(cpu);
		DEBUGASSERT(rtcb != NULL);

		/* Do we already have interrupts disabled? */

		if (rtcb->irqcount > 0) {
			/* Yes... make sure that the spinlock is set and increment the
			 * IRQ lock count.
			 *
			 * NOTE: If irqcount > 0 then (1) we are in a critical section,
			 * and (2) this CPU should hold the lock.
			 */

			DEBUGASSERT(spin_islocked(&g_cpu_irqlock) && \
					(g_cpu_irqset & (1 << this_cpu())) != 0 && \
					rtcb->irqcount < INT16_MAX);
			rtcb->irqcount++;
		} else {
			/* If we get here with irqcount == 0, then we know that the
			 * current task running on this CPU is not in a critical
			 * section.  However other tasks on other CPUs may be in a
			 * critical section.  If so, we must wait until they release
			 * the spinlock.
			 */

			DEBUGASSERT((g_cpu_irqset & (1 << cpu)) == 0);

			if (!irq_waitlock(cpu)) {
				/* We are in a deadlock condition due to a pending pause
				 * request interrupt.  Re-enable interrupts on this CPU
				 * and try again.  Briefly re-enabling interrupts should
				 * be sufficient to permit processing the pending pause
				 * request.
				 */

				irqrestore(ret);
				goto try_again;
			}

			/* Then set the lock count to 1.
			 *
			 * Interrupts disables must follow a stacked order.  We
			 * cannot other context switches to re-order the enabling
			 * disabling of interrupts.
			 *
			 * The scheduler accomplishes this by treating the irqcount
			 * like lockcount:  Both will disable pre-emption.
			 */

			spin_setbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, 
					&g_cpu_irqlock);
			rtcb->irqcount = 1;

			/* Note that we have entered the critical section */

#ifdef CONFIG_SCHED_INSTRUMENTATION_CSECTION
			sched_note_csection(rtcb, true);
#endif
		}
	}
	}

	/* Return interrupt status */
	return ret;
}

#else /* CONFIG_SMP */

irqstate_t enter_critical_section(void)
{
	irqstate_t ret;

	/* Disable interrupts */

	ret = irqsave();

	/* Check if we were called from an interrupt handler and that the task
	 * lists have been initialized.
	 */

	if (!up_interrupt_context() && g_os_initstate >= OSINIT_TASKLISTS) {
		FAR struct tcb_s *rtcb = this_task();
		DEBUGASSERT(rtcb != NULL);

		/* Have we just entered the critical section?  Or is this a nested
		 * call to enter_critical_section.
		 */

		DEBUGASSERT(rtcb->irqcount >= 0 && rtcb->irqcount < INT16_MAX);
		if (++rtcb->irqcount == 1) {
			/* Note that we have entered the critical section */

#ifdef CONFIG_SCHED_INSTRUMENTATION_CSECTION
			sched_note_csection(rtcb, true);
#endif
		}
	}

	/* Return interrupt status */
	return ret;
}
#endif /* CONFIG_SMP */

/****************************************************************************
 * Name: leave_critical_section
 *
 * Description:
 *   Decrement the IRQ lock count and if it decrements to zero then release
 *   the spinlock.
 *
 ****************************************************************************/

#ifdef CONFIG_SMP
void leave_critical_section(irqstate_t flags)
{
	int cpu;

	/* Verify that the system has sufficiently initialized so that the task
	 * lists are valid.
	 */

	if (g_os_initstate >= OSINIT_TASKLISTS) {
		/* If called from an interrupt handler, then just release the
		 * spinlock.  The interrupt handling logic should already hold the
		 * spinlock if enter_critical_section() has been called.  Unlocking
		 * the spinlock will allow interrupt handlers on other CPUs to execute
		 * again.
		 */

		if (up_interrupt_context()) {
			/* We are in an interrupt handler. Check if the last call to
			 * enter_critical_section() was nested.
			 */

			cpu = this_cpu();
			if (g_cpu_nestcount[cpu] > 1) {
				/* Yes.. then just decrement the nesting count */

				DEBUGASSERT(spin_islocked(&g_cpu_irqlock));
				g_cpu_nestcount[cpu]--;
			} else {
				/* No, not nested. Restore the g_cpu_irqset for this CPU
				 * and release the spinlock (if necessary).
				 */

				DEBUGASSERT(spin_islocked(&g_cpu_irqlock) && \
						g_cpu_nestcount[cpu] == 1);

				FAR struct tcb_s *rtcb = current_task(cpu);
				DEBUGASSERT(rtcb != NULL);

				if (rtcb->irqcount <= 0) {
					spin_clrbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, 
							&g_cpu_irqlock);
				}

				g_cpu_nestcount[cpu] = 0;
			}
		} else {
			FAR struct tcb_s *rtcb;

			/* Get the TCB of the currently executing task on this CPU (avoid
			 * using this_task() which can recurse.
			 */

			cpu  = this_cpu();
			rtcb = current_task(cpu);
			DEBUGASSERT(rtcb != NULL && rtcb->irqcount > 0);

			/* Normal tasking context.  We need to coordinate with other
			 * tasks.
			 *
			 * Will we still have interrupts disabled after decrementing the
			 * count?
			 */

			if (rtcb->irqcount > 1) {
				/* Yes... the spinlock should remain set */

				DEBUGASSERT(spin_islocked(&g_cpu_irqlock));
				rtcb->irqcount--;
			} else {
				/* No.. Note that we have left the critical section */

#ifdef CONFIG_SCHED_INSTRUMENTATION_CSECTION
				sched_note_csection(rtcb, false);
#endif
				/* Decrement our count on the lock.  If all CPUs have
				 * released, then unlock the spinlock.
				 */


				DEBUGASSERT(spin_islocked(&g_cpu_irqlock) && 
						(g_cpu_irqset & (1 << cpu)) != 0);

				/* Check if releasing the lock held by this CPU will unlock the
				 * critical section.
				 */

				if ((g_cpu_irqset & ~(1 << cpu)) == 0) {
					/* Yes.. Check if there are pending tasks and that pre-
					 * emption is also enabled.  This is necessary because we
					 * may have deferred the up_release_pending() call in
					 * sched_unlock() because we were within a critical
					 * section then.
					 */

					if (g_pendingtasks.head != NULL && 
						!sched_islocked_global()) {
						/* Release any ready-to-run tasks that have collected
						 * in g_pendingtasks.  NOTE: This operation has a very
						 * high likelihood of causing this task to be switched
						 * out!
						 */

						up_release_pending();
					}
				}

				/* Now, possibly on return from a context switch, clear our
				 * count on the lock.  If all CPUs have released the lock,
				 * then unlock the global IRQ spinlock.
				 */

				rtcb->irqcount = 0;
				spin_clrbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, 
						&g_cpu_irqlock);

				/* Have all CPUs released the lock? */
			}
		}
	}

	/* Restore the previous interrupt state which may still be interrupts
	 * disabled (but we don't have a mechanism to verify that now)
	 */

	irqrestore(flags);
}

#else

void leave_critical_section(irqstate_t flags)
{
	/* Check if we were called from an interrupt handler and that the tasks
	 * lists have been initialized.
	 */

	if (!up_interrupt_context() && g_os_initstate >= OSINIT_TASKLISTS) {
		FAR struct tcb_s *rtcb = this_task();
		DEBUGASSERT(rtcb != NULL);

		/* Have we left entered the critical section?  Or are we still
		 * nested.
		 */

		DEBUGASSERT(rtcb->irqcount > 0);
		if (--rtcb->irqcount <= 0) {
			/* Note that we have left the critical section */

#ifdef CONFIG_SCHED_INSTRUMENTATION_CSECTION
			sched_note_csection(rtcb, false);
#endif
		}
	}

	/* Restore the previous interrupt state. */

	irqrestore(flags);
}
#endif

/****************************************************************************
 * Name:  irq_cpu_locked
 *
 * Description:
 *   Test if the IRQ lock set OR if this CPU holds the IRQ lock
 *   There is an interaction with pre-emption controls and IRQ locking:
 *   Even if the pre-emption is enabled, tasks will be forced to pend if
 *   the IRQ lock is also set UNLESS the CPU starting the task is the
 *   holder of the IRQ lock.
 *
 * Input Parameters:
 *   rtcb - Points to the blocked TCB that is ready-to-run
 *
 * Returned Value:
 *   true  - IRQs are locked by a different CPU.
 *   false - IRQs are unlocked OR if they are locked BUT this CPU
 *           is the holder of the lock.
 *
 *   Warning: This values are volatile at only valid at the instance that
 *   the CPU set was queried.
 *
 ****************************************************************************/

#ifdef CONFIG_SMP
bool irq_cpu_locked(int cpu)
{
	cpu_set_t irqset;

	/* g_cpu_irqset is not valid in early phases of initialization */

	if (g_os_initstate < OSINIT_OSREADY) {
		/* We are still single threaded.  In either state of g_cpu_irqlock,
		 * the correct return value should always be false.
		 */

		return false;
	}

	/* Test if g_cpu_irqlock is locked.  We don't really need to use check
	 * g_cpu_irqlock to do this, we can use the g_cpu_set.
	 *
	 * Sample the g_cpu_irqset once.  That is an atomic operation.  All
	 * subsequent operations will operate on the sampled cpu set.
	 */

	irqset = (cpu_set_t)g_cpu_irqset;
	if (irqset != 0) {
		/* Some CPU holds the lock.  So g_cpu_irqlock should be locked.
		 * Return false if the 'cpu' is the holder of the lock; return
		 * true if g_cpu_irqlock is locked, but this CPU is not the
		 * holder of the lock.
		 */

		return ((irqset & (1 << cpu)) == 0);
	} else {
		/* No CPU holds the lock,
		 * In this case g_cpu_irqlock should be unlocked.  However, if
		 * the lock was established in the interrupt handler AND there are
		 * no bits set in g_cpu_irqset, that probably means only that
		 * critical section was established from an interrupt handler.
		 * Return false in either case.
		 */

		return false;
	}
}
#endif

#ifdef CONFIG_SMP
void restore_critical_section(void)
{
	/* NOTE: The following logic for adjusting global IRQ controls were
	 * derived from sched_add_readytorun() and sched_removedreadytorun()
	 * Here, we only handles clearing logic to defer unlocking IRQ lock
	 * followed by context switching.
	 */

	FAR struct tcb_s *tcb = this_task();
	int me = this_cpu();

	/* Adjust global IRQ controls.  If irqcount is greater than zero,
	 * then this task/this CPU holds the IRQ lock
	 */

	if (tcb->irqcount > 0)
	{
		/* Do notihing here
		 * NOTE: spin_setbit() is done in sched_add_readytorun()
		 * and sched_remove_readytorun()
		 */
	}

	/* No.. This CPU will be relinquishing the lock.  But this works
	 * differently if we are performing a context switch from an
	 * interrupt handler and the interrupt handler has established
	 * a critical section.  We can detect this case when
	 * g_cpu_nestcount[me] > 0.
	 */

	else if (g_cpu_nestcount[me] <= 0)
	{
		/* Release our hold on the IRQ lock. */

		if ((g_cpu_irqset & (1 << me)) != 0)
		{
			spin_clrbit(&g_cpu_irqset, me, &g_cpu_irqsetlock,
			&g_cpu_irqlock);
		}
	}
}
#endif /* CONFIG_SMP */

#endif /* CONFIG_IRQCOUNT */
